SherbyNet - /page/wikibook/entry.mu

<!DOCTYPE html>

<title>Entropy rate</title>

</head>

<span id="openzim-page-title" class="mw-page-title-main"><span class="mw-page-title-main">Entropy rate</span></span>

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar nomobile nowraplinks hlist"><tbody><tr><th class="sidebar-title"><a href="Information_theory" title="Information theory">Information theory</a></th></tr><tr><td class="sidebar-image"><span typeof="mw:File"></span></td></tr><tr><td class="sidebar-content">

<ul><li><a href="Entropy_(information_theory)" title="Entropy (information theory)">Entropy</a></li>

<li><a href="Differential_entropy" title="Differential entropy">Differential entropy</a></li>

<li><a href="Conditional_entropy" title="Conditional entropy">Conditional entropy</a></li>

<li><a href="Joint_entropy" title="Joint entropy">Joint entropy</a></li>

<li><a href="Mutual_information" title="Mutual information">Mutual information</a></li>

<li><a href="Directed_information" title="Directed information">Directed information</a></li>

<li><a href="Conditional_mutual_information" title="Conditional mutual information">Conditional mutual information</a></li>

<li><a href="Relative_entropy" class="mw-redirect" title="Relative entropy">Relative entropy</a></li>

<li><a href="Limiting_density_of_discrete_points" title="Limiting density of discrete points">Limiting density of discrete points</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Asymptotic_equipartition_property" title="Asymptotic equipartition property">Asymptotic equipartition property</a></li>

<li><a href="Rate%E2%80%93distortion_theory" title="Rate–distortion theory">Rate–distortion theory</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Shannon's_source_coding_theorem" title="Shannon's source coding theorem">Shannon's source coding theorem</a></li>

<li><a href="Channel_capacity" title="Channel capacity">Channel capacity</a></li>

<li><a href="Noisy-channel_coding_theorem" title="Noisy-channel coding theorem">Noisy-channel coding theorem</a></li>

<li><a href="Shannon%E2%80%93Hartley_theorem" title="Shannon–Hartley theorem">Shannon–Hartley theorem</a></li></ul></td>

</tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

<p>In the mathematical theory of <a href="Probability" title="Probability">probability</a>, the <b>entropy rate</b> or <b>source information rate</b> is a function assigning an <a href="Entropy_(information_theory)" title="Entropy (information theory)">entropy</a> to a <a href="Stochastic_process" title="Stochastic process">stochastic process</a>.

</p><p>For a <a href="Strongly_stationary" class="mw-redirect" title="Strongly stationary">strongly stationary</a> process, the <a href="Conditional_entropy" title="Conditional entropy">conditional entropy</a> for latest random variable eventually tend towards this rate value.

</p>

<div class="mw-heading mw-heading2"><h2 id="Definition">Definition</h2></div>

<p>A process <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X}</annotation>

</semantics>

</math></span><img src="./68baa052181f707c662844a465bfeeb135e82bab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}" loading="lazy"></span> with a <a href="Countable" class="mw-redirect" title="Countable">countable</a> index gives rise to the sequence of its <a href="Joint_entropy" title="Joint entropy">joint entropies</a> <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle H_{n}(X_{1},X_{2},\dots X_{n})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle H_{n}(X_{1},X_{2},\dots X_{n})}</annotation>

</semantics>

</math></span><img src="./fe14ee27914aeb4e2572b2068a35ca746430d384.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.237ex; height:2.843ex;" alt="{\displaystyle H_{n}(X_{1},X_{2},\dots X_{n})}" loading="lazy"></span>. If the limit exists, the entropy rate is defined as

</p>

</mrow>

</munder>

<mfrac>

</mfrac>

</mstyle>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle H(X):=\lim _{n\to \infty }{\tfrac {1}{n}}H_{n}.}</annotation>

</semantics>

</math></span><img src="./d04f4f92f2e316af3cab3954a3bd535b2bbdeadd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; width:19.877ex; height:4.176ex;" alt="{\displaystyle H(X):=\lim _{n\to \infty }{\tfrac {1}{n}}H_{n}.}" loading="lazy"></span></dd></dl>

<p>Note that given any sequence <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (a_{n})_{n}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (a_{n})_{n}}</annotation>

</semantics>

</math></span><img src="./cc87f936c2f7d77029ef0eb03494fc8533b123bf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.476ex; height:2.843ex;" alt="{\displaystyle (a_{n})_{n}}" loading="lazy"></span> with <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle a_{0}=0}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle a_{0}=0}</annotation>

</semantics>

</math></span><img src="./e8f3589226b1f07bd27b7c82d8f470a4685fffe2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.545ex; height:2.509ex;" alt="{\displaystyle a_{0}=0}" loading="lazy"></span> and letting <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \Delta a_{k}:=a_{k}-a_{k-1}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Delta a_{k}:=a_{k}-a_{k-1}}</annotation>

</semantics>

</math></span><img src="./a5d58fdf88466a7a19626e89fabdbbf4e0dbcbe5.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:17.578ex; height:2.509ex;" alt="{\displaystyle \Delta a_{k}:=a_{k}-a_{k-1}}" loading="lazy"></span>, by <a href="Telescoping_series" title="Telescoping series">telescoping</a> one has <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle a_{n}={\textstyle \sum _{k=1}^{n}}\Delta a_{k}}">

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</munderover>

</mstyle>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle a_{n}={\textstyle \sum _{k=1}^{n}}\Delta a_{k}}</annotation>

</semantics>

</math></span><img src="./52b38a6e0da85da9b86e322e98ce98e8bebece1c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:15.444ex; height:3.176ex;" alt="{\displaystyle a_{n}={\textstyle \sum _{k=1}^{n}}\Delta a_{k}}" loading="lazy"></span>. The entropy rate thus computes the mean of the first <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n}</annotation>

</semantics>

</math></span><img src="./a601995d55609f2d9f5e233e36fbe9ea26011b3b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.395ex; height:1.676ex;" alt="{\displaystyle n}" loading="lazy"></span> such entropy changes, with <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n}</annotation>

</semantics>

</math></span><img src="./a601995d55609f2d9f5e233e36fbe9ea26011b3b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.395ex; height:1.676ex;" alt="{\displaystyle n}" loading="lazy"></span> <a href="Limit_(mathematics)" title="Limit (mathematics)">going to</a> <a href="Infinity" title="Infinity">infinity</a>.

The <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n}</annotation>

</semantics>

</math></span><img src="./a601995d55609f2d9f5e233e36fbe9ea26011b3b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.395ex; height:1.676ex;" alt="{\displaystyle n}" loading="lazy"></span>th entropy change is itself the <a href="Conditional_entropy" title="Conditional entropy">conditional entropy</a> <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle H(X_{n}|X_{n-1},X_{n-2},...)}">

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle H(X_{n}|X_{n-1},X_{n-2},...)}</annotation>

</semantics>

</math></span><img src="./c8889e4b144a9a3aadee51ccd60a85d10b9b4647.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:23.319ex; height:2.843ex;" alt="{\displaystyle H(X_{n}|X_{n-1},X_{n-2},...)}" loading="lazy"></span>. The entropy rate is thus the average entropy of the distribution of the <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n}</annotation>

</semantics>

</math></span><img src="./a601995d55609f2d9f5e233e36fbe9ea26011b3b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.395ex; height:1.676ex;" alt="{\displaystyle n}" loading="lazy"></span>th variable once the previous ones are known. The behaviour of joint entropies from one index to the next is also explicitly subject in some <a href="Entropy_(information_theory)#Alternative_characterization" title="Entropy (information theory)">characterizations of entropy</a>.

</p>

<div class="mw-heading mw-heading2"><h2 id="Discussion">Discussion</h2></div>

<p>While <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X}</annotation>

</semantics>

</math></span><img src="./68baa052181f707c662844a465bfeeb135e82bab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}" loading="lazy"></span> may be understood as a sequence of random variables, the entropy rate <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle H(X)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle H(X)}</annotation>

</semantics>

</math></span><img src="./bd232b6fb5ea803efc1154d2efb0c3fe00a4531b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.853ex; height:2.843ex;" alt="{\displaystyle H(X)}" loading="lazy"></span> represents the average entropy change per one random variable, in the long term.

</p><p>It can be thought of as a general property of stochastic sources - this is the subject of the <a href="Asymptotic_equipartition_property" title="Asymptotic equipartition property">asymptotic equipartition property</a>.

</p>

<div class="mw-heading mw-heading3"><h3 id="For_strongly_stationary_processes">For strongly stationary processes</h3></div>

<p>A stochastic process also gives rise to a sequence of conditional entropies, comprising more and more random variables.

For strongly stationary stochastic processes, the entropy rate equals the limit of that sequence

</p>

</mrow>

</munder>

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle H(X)=\lim _{n\to \infty }H(X_{n}|X_{n-1},X_{n-2},\dots X_{1})}</annotation>

</semantics>

</math></span><img src="./1051169fb8e94a2609b4f29a7dd85ef73de6752a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; width:39.917ex; height:3.843ex;" alt="{\displaystyle H(X)=\lim _{n\to \infty }H(X_{n}|X_{n-1},X_{n-2},\dots X_{1})}" loading="lazy"></span></dd></dl>

<p>The quantity given by the limit on the right is also denoted <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle H'(X)}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle H'(X)}</annotation>

</semantics>

</math></span><img src="./a33e34fd27aef76f130f86d0b558fabce3d2e6a1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.577ex; height:3.009ex;" alt="{\displaystyle H'(X)}" loading="lazy"></span>, which is motivated to the extent that here this is then again a rate associated with the process, in the above sense.

</p>

<div class="mw-heading mw-heading3"><h3 id="For_Markov_chains">For Markov chains</h3></div>

<p>Since a stochastic process defined by a <a href="Markov_chain" title="Markov chain">Markov chain</a> that is <a href="Markov_chain#Properties" title="Markov chain">irreducible</a> and <a href="Aperiodic" class="mw-redirect" title="Aperiodic">aperiodic</a> has a <a href="Stationary_distribution" title="Stationary distribution">stationary distribution</a>, the entropy rate is independent of the initial distribution.<sup id="cite_ref-1" class="reference"><a href="#cite_note-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup>

</p><p>For example, consider a Markov chain defined on a <a href="Countable" class="mw-redirect" title="Countable">countable</a> number of states. Given its <a href="Stochastic_matrix" title="Stochastic matrix">right stochastic transition matrix</a> <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{ij}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle P_{ij}}</annotation>

</semantics>

</math></span><img src="./43ef37c239b6d38f1e951a31eb1a3bd295271b40.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.969ex; height:2.843ex;" alt="{\displaystyle P_{ij}}" loading="lazy"></span> and an entropy

</p>

<msub>

</mrow>

</msub>

</mrow>

</munder>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{i}:=-\sum _{j}P_{ij}\log P_{ij}}</annotation>

</semantics>

</math></span><img src="./5804912c6066759f3ec3d74d17bf849e8397b461.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:21.506ex; height:5.843ex;" alt="{\displaystyle h_{i}:=-\sum _{j}P_{ij}\log P_{ij}}" loading="lazy"></span></dd></dl>

<p>associated with each state, one finds

</p>

</mrow>

</munder>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \displaystyle H(X)=\sum _{i}\mu _{i}h_{i},}</annotation>

</semantics>

</math></span><img src="./f8cbdf474c9f9121cd3d6264527d3b07ce196402.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:17.68ex; height:5.509ex;" alt="{\displaystyle \displaystyle H(X)=\sum _{i}\mu _{i}h_{i},}" loading="lazy"></span></dd></dl>

<p>where <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mu _{i}}</annotation>

</semantics>

</math></span><img src="./dea0a0293841cce9eef98b55e53a92b82ae59ee4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.201ex; height:2.176ex;" alt="{\displaystyle \mu _{i}}" loading="lazy"></span> is the <a href="Asymptotic_distribution" title="Asymptotic distribution">asymptotic distribution</a> of the chain.

</p><p>In particular, it follows that the entropy rate of an <a href="Independent_and_identically_distributed" class="mw-redirect" title="Independent and identically distributed">i.i.d.</a> <a href="Stochastic_process" title="Stochastic process">stochastic process</a> is the same as the entropy of any individual member in the process.

</p>

<div class="mw-heading mw-heading3"><h3 id="For_hidden_Markov_models">For hidden Markov models</h3></div>

<p>The entropy rate of hidden Markov models (HMM) has no known closed-form solution. However, it has known upper and lower bounds. Let the underlying Markov chain <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X_{1:\infty }}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X_{1:\infty }}</annotation>

</semantics>

</math></span><img src="./eae4e1421c3410b88430aecc99b73c580aeec1f7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:5.079ex; height:2.509ex;" alt="{\displaystyle X_{1:\infty }}" loading="lazy"></span> be stationary, and let <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y_{1:\infty }}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Y_{1:\infty }}</annotation>

</semantics>

</math></span><img src="./58f847201c38329814b811356286b728b1ff1688.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.505ex; height:2.509ex;" alt="{\displaystyle Y_{1:\infty }}" loading="lazy"></span> be the observable states, then we have<span class="mwe-math-element mwe-math-element-block"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle H(Y_{n}|X_{1},Y_{1:n-1})\leq H(Y)\leq H(Y_{n}|Y_{1:n-1})}">

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle H(Y_{n}|X_{1},Y_{1:n-1})\leq H(Y)\leq H(Y_{n}|Y_{1:n-1})}</annotation>

</semantics>

</math></span></span>and at the limit of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n\to \infty }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n\to \infty }</annotation>

</semantics>

</math></span><img src="./a0d55d9b32f6fa8fab6a84ea444a6b5a24bb45e1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.333ex; height:1.843ex;" alt="{\displaystyle n\to \infty }" loading="lazy"></span>, both sides converge to the middle.<sup id="cite_ref-2" class="reference"><a href="#cite_note-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading2"><h2 id="Applications">Applications</h2></div>

<p>The entropy rate may be used to estimate the complexity of stochastic processes. It is used in diverse applications ranging from characterizing the complexity of languages, blind source separation, through to optimizing quantizers and data compression algorithms. For example, a maximum entropy rate criterion may be used for <a href="Feature_selection" title="Feature selection">feature selection</a> in <a href="Machine_learning" title="Machine learning">machine learning</a>.<sup id="cite_ref-3" class="reference"><a href="#cite_note-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup>

</p>

<ul><li><a href="Information_source_(mathematics)" title="Information source (mathematics)">Information source (mathematics)</a></li>

<li><a href="Markov_information_source" title="Markov information source">Markov information source</a></li>

<li><a href="Asymptotic_equipartition_property" title="Asymptotic equipartition property">Asymptotic equipartition property</a></li>

<li><a href="Maximal_entropy_random_walk" title="Maximal entropy random walk">Maximal entropy random walk</a> - chosen to maximize entropy rate</li></ul>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist">

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFCoverThomas2006" class="citation book cs1">Cover, Thomas M.; Thomas, Joy A. (2006). <i>Elements of information theory</i> (2nd ed.). Hoboken, N.J: Wiley-Interscience. p. 78. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-471-24195-9</bdi>.</cite></span>

</li>

<li id="cite_note-2"><span class="mw-cite-backlink"><b><a href="#cite_ref-2">^</a></b></span> <span class="reference-text"><cite id="CITEREFCoverThomas2006" class="citation book cs1">Cover, Thomas M.; Thomas, Joy A. (2006). "4.5. Functions of Markov chains". <i>Elements of information theory</i> (2nd ed.). Hoboken, N.J: Wiley-Interscience. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-471-24195-9</bdi>.</cite></span>

</li>

<li id="cite_note-3"><span class="mw-cite-backlink"><b><a href="#cite_ref-3">^</a></b></span> <span class="reference-text"><cite id="CITEREFEinicke2018" class="citation journal cs1">Einicke, G. A. (2018). "Maximum-Entropy Rate Selection of Features for Classifying Changes in Knee and Ankle Dynamics During Running". <i>IEEE Journal of Biomedical and Health Informatics</i>. <b>28</b> (4): <span class="nowrap">1097–</span>1103. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2501.13750">2501.13750</a></span>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FJBHI.2017.2711487">10.1109/JBHI.2017.2711487</a>. <a href="Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://hdl.handle.net/10810%2F68978">10810/68978</a></span>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/29969403">29969403</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:49555941">49555941</a>.</cite></span>

</li>

</ol></div></div>

<div class="mw-heading mw-heading2"><h2 id="External_links">External links</h2></div>

<ul><li><a rel="nofollow" class="external text" href="http://staff.ustc.edu.cn/~cgong821/Wiley.Interscience.Elements.of.Information.Theory.Jul.2006.eBook-DDU.pdf">Cover, T. and Thomas, J. Elements of Information Theory. John Wiley and Sons, Inc. Second Edition, 2006.</a></li></ul></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-07-09" href="https://en.wikipedia.org/wiki/?title=Entropy_rate&oldid=1299565422">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>